# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy.exceptions import DropItem
import codecs


class BaikeAvpPipeline(object):
    """
    保存词条三元组的Pipeline，将每个页面的的三元组追加写入到baike_avp.txt。
    在setting.py中设置的优先级为200（值越大，优先级越低）。
    """

    def __init__(self):
        self.file = codecs.open('baike_avp.txt', 'ab', 'utf-8')

    def process_item(self, item, spider):
        template = "<%s> <%s> <%s> .\n"
        line = template % ("".join(item['subject']), ",".join(item['guest']), ",".join(item['relation']))
        self.file.write(line)
        return item


class DuplicatesPipeline(object):
    """
    对于爬取到的三元组进行去重判断，若已经在本次爬取过程中获取过，则忽略该三元组。
    在setting.py中设置的优先级为100，即先进行去重，再保存三元组。
    """

    def __init__(self):
        self.item_seen = set()

    def process_item(self, item, spider):
        triple_tuple = (str(item['subject']), str(item['relation']), str(item['guest']))
        if triple_tuple in self.item_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.item_seen.add(triple_tuple)
            return item